#爬取豆瓣图书top250
import os,time,requests,json
from pyquery import PyQuery
from urllib.request import urlretrieve

def getPage(offset):
    '''获取一页的网页信息'''
    headers = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.26 Safari/537.36 Core/1.63.6788.400 QQBrowser/10.3.2767.400',}
    url = 'https://book.douban.com/top250?start='+str(offset)
    try:
        res = requests.get(url,headers=headers)
        if res.status_code == 200:
            return res.text
        else:
            print('没有爬取到数据')
            return None
    except Exception as e:
        print(e)
        return None

def parsePage(html):
    '''解析图书信息'''
    #解析html文档，获取PyQuery对象
    doc = PyQuery(html)
    #获取网页中的所有标签，并遍历输出标签名
    items = doc('tr.item')
    #遍历封装数据并返回
    try:
        for item in items.items():
            yield{
                'image':item.find('a.nbg img').attr('src'),
                'title':item.find('td[valign="top"] div.pl2 a').attr('title'),
                'author':item.find("p.pl").text(),
                'score':item.find("span.rating_nums").text(),
                'words':item.find("span.inq").text(),
            }
    except Exception as e:
        # print(e)
        return None

def writeFile(item):
    '''储存图书信息'''
    #获取图片
    imgurl = item.get('image')
    title = item.get('title')
    urlretrieve(imgurl,'./images/douban/pyquery/'+title+'.jpg')
    #将图书详细信息写入文件
    with open('./douban_pyquery.txt','a',encoding='utf-8') as f:
        f.write(json.dumps(item,ensure_ascii=False)+'\n')

def main(offset):
    '''主函数'''
    #判断当前路径是否存在images文件夹
    if not os.path.exists('./images/douban/pyquery'):
        os.makedirs('./images/douban/pyquery')
    html = getPage(offset)
    items = parsePage(html)
    for item in items:
        # print(item)
        writeFile(item)

if __name__ == '__main__':
    # main(0)
    for i in range(10):
        offset = i*25
        main(offset)
        print('第%s条数据完成。'%((i+1)*25))
        time.sleep(1)